import json
import urllib
import requests

from urllib import parse
from fake_useragent import UserAgent
from pyquery import PyQuery as pq


def sendXHR(url, headers={}):
    response = requests.get(url, headers=headers)
    result = response.content.decode()
    return json.loads(result)


def getDocByUrl(url, headers={}):
    headers['user-agent'] = getUserAgent()
    req = urllib.request.Request(url=url, headers=headers)
    page = urllib.request.urlopen(req)
    charset = page.headers['Content-Type'].lower().split("charset=")[1]
    html = page.read().decode(charset, 'ignore').encode("utf-8", 'ignore')
    return pq(html)

def getUserAgent():
    ua = UserAgent()
    return ua.random

def getEntryIdByUrl(url,subIndex = 0):
    urlList = url.split('/')
    index = len(urlList) - 1 - subIndex
    articleName = urlList[index]
    if articleName.find('.') < 0:
        return articleName
    return articleName[0:articleName.find('.')]

def getEntryIdByQuery(url,query):
    params = parse.parse_qs(parse.urlparse(url).query)
    return ''.join(params[query])
